package crawler.proxy.impl;

import crawler.proxy.Proxy;
import crawler.proxy.ProxySpider;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
 * USER: lintc 【lintiancong@zhuojianchina.com】
 * DATE: 2017-09-14 10:42
 */
public class KuaiProxy extends ProxySpider{

    public KuaiProxy() {
        super("http://www.kuaidaili.com/free/inha/index");
    }

    @Override
    public Set<Proxy> getList(int index) {
        Set<Proxy> list = new HashSet<>();
//        catchUrl = catchUrl.replace("index", index + "");
        Document document = null;
        try {
            document = Jsoup.connect(catchInvokeUrl).
                    userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:55.0) Gecko/20100101 Firefox/55.0").
                    header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8").
                    header("Accept-Encoding", "gzip, deflate").
                    header("Accept-Language","zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3").
                    header("Cookie", "yd_cookie=4e8a3b20-dbfd-4a5ef160015b3919dc269a2516aeed619b1d; _ydclearance=0c513b7eb13ce67d647527f6-6895-4d02-ba67-a265fa467b40-1505358594; channelid=0; sid=1505351724680153; _ga=GA1.2.1372376191.1505351397; _gid=GA1.2.566373600.1505351397; Hm_lvt_7ed65b1cc4b810e9fd37959c9bb51b31=1505351397; Hm_lpvt_7ed65b1cc4b810e9fd37959c9bb51b31=1505351606").
                    header("Content-Type", "application/json;charset=UTF-8").get();
            Elements elements = document.select("tr:gt(0)");
            Proxy proxy;
            for (Element element : elements) {
                proxy = new Proxy();
                String ip = element.select("td:eq(0)").first().text();
                String port  = element.select("td:eq(1)").first().text();
                String isAnonymous = element.select("td:eq(2)").first().text();
                if (isAnonymous.contains(filterStr)) {
                    proxy.setIp(ip);
                    proxy.setPort(port);
                    list.add(proxy);
                }
            }
        } catch (IOException e) {
            LOG.error(catchInvokeUrl + ": 抓取失败 (" + e.getMessage() + ")");
        }
        return list;
    }
}
